{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c8512c05-37e1-4a53-9ce8-79cf4fae8637",
   "metadata": {},
   "source": [
    "# 0 准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d5088f74-37c1-453e-9e45-33ba09ea1ba0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pytz\n",
    "from datetime import datetime\n",
    "import json, re, gc\n",
    "import glob\n",
    "import os, sys, rarfile\n",
    "from tqdm import tqdm\n",
    "# from typing import Dict, Any\n",
    "from pathlib import Path\n",
    "\n",
    "import platform\n",
    "import jaydebeapi\n",
    "import jpype\n",
    "import jpype.imports\n",
    "from jpype.types import *\n",
    "import subprocess\n",
    "import pyodbc, io\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "import torch\n",
    "import pickle\n",
    "# from openai import OpenAI\n",
    "import warnings, logging\n",
    "sys.path.append('./')\n",
    "\n",
    "from utils.load_access_data import *\n",
    "from utils.file_unrar import *\n",
    "# from utils.data_mapping import *\n",
    "from utils.data_mapping_36country2 import *\n",
    "from utils.invoke_llm import *\n",
    "from utils.file_process import *\n",
    "from utils.logging_config import setup_logger\n",
    "\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "logger = setup_logger('Data Goverance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "55fa8a75-a6c5-4dd0-9125-c0d997eb3b8c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['all_country_cols_dict', 'all_country_list', 'llm_country_describe', 'llm_country_mapping', 'country_mapping', 'currency_df', 'currency_on_usd'])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# model_result = pickle.load(open('result/model_rslt_03_20250810.pickle', 'rb'))\n",
    "model_result = pickle.load(open('result/model_rslt_05_20250810.pickle', 'rb'))\n",
    "model_result.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "869dd29a-0087-4e8a-84de-d559e46e7c4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_country_mapping = model_result['llm_country_mapping']\n",
    "all_data_country_list = model_result['all_country_list']\n",
    "country_mapping = model_result['country_mapping']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5569560d-4cc8-4ab2-913e-cf9b67b94797",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载国家映射表\n",
    "country_code = pd.read_csv('param_files/国家代码表.csv')\n",
    "country_code_dict = country_code.loc[:,['英文国名','中文国名']].to_dict('records')\n",
    "country_code_dict_detail = country_code.to_dict('records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf760a37-0ad9-4db3-a0ea-533f898e44e0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a62298da-70da-4988-95db-8d9bef6ff4e4",
   "metadata": {},
   "source": [
    "# 1 利用LLM做国家标准化"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43ceb39b-107c-4b3b-807b-a943876b3020",
   "metadata": {},
   "source": [
    "## 1.1 获取全部国家列表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad87b575-57c3-4a61-a2b8-d6d6b03fd0fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_result.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f32cfd0-bedc-4eb4-98a8-43c0fa6f1309",
   "metadata": {},
   "outputs": [],
   "source": [
    "root_path = Path('dataset/data_20250724/20251t3_csv_save2')\n",
    "_, file_paths = get_dir_files(root_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b85cf368-b63d-477c-91fe-4f7539929af4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# file_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c66350b0-501b-4d05-90ba-76a32304db98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get_all_data_countries=get_all_data_countries\n",
    "print(len(file_paths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "500d88c6-34ba-4cbb-ab4d-5c8a303108db",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# start = time.time()\n",
    "\n",
    "# params_list = [([x],) for x in file_paths]\n",
    "# results = []\n",
    "# with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "#     # 提交多个任务\n",
    "#     futures = [\n",
    "#         executor.submit(lambda p: get_all_data_countries(*p), params)\n",
    "#         for params in params_list\n",
    "#     ]\n",
    "#     # 按完成顺序获取结果\n",
    "#     for future in as_completed(futures):\n",
    "#         res = future.result()\n",
    "#         results.append(res)\n",
    "#         logger.info(f'future.result: {len(res)}')\n",
    "\n",
    "# cntry_lst_tmp = [x for r in results for x in r]\n",
    "# all_data_country_list = list(set(cntry_lst_tmp))\n",
    "# print(len(all_data_country_list))\n",
    "# # model_result['all_country_list'] = all_data_country_list\n",
    "# # pickle.dump(model_result, open('result/model_rslt_01.pickle', 'wb'))\n",
    "\n",
    "# end = time.time()\n",
    "# logger.info(f'country_normal_mapping takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3093eafd-0f05-4352-b778-a06dbc30e6d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(all_data_country_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68739d33-b3bf-4211-906b-7403488e1f95",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 单线程\n",
    "# # 获取数据中所有国家列表\n",
    "# all_data_country_list = get_all_data_countries(file_paths)\n",
    "# print(len(all_data_country_list))\n",
    "# model_result['all_country_list'] = all_data_country_list\n",
    "# pickle.dump(model_result, open('result/model_rslt_01.pickle', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "790b8007-1a5e-4e4c-86aa-a7dba7fe2e0b",
   "metadata": {},
   "source": [
    "## 1.2 LLM对国家名称做标准化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f1c5b56-986a-4e30-8576-b2fde86e203d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 轻易不要跑，比较慢 - 需要半小时\n",
    "# start = time.time()\n",
    "# batch=20\n",
    "# step=5\n",
    "# STP = batch*step\n",
    "# all_country_list_2d = [all_data_country_list[i:i+STP] for i in range(0, len(all_data_country_list), STP)]\n",
    "# logger.info(f'all_country_list_2d length : {len(all_country_list_2d)}')\n",
    "\n",
    "# params_list = [(x, country_code_dict, batch, step) for x in all_country_list_2d]\n",
    "\n",
    "# results = []\n",
    "# with ThreadPoolExecutor(max_workers=20) as executor:\n",
    "#     # 提交多个任务\n",
    "#     futures = [\n",
    "#         executor.submit(lambda p: get_llm_country_normal2(*p), params)\n",
    "#         for params in params_list\n",
    "#     ]\n",
    "#     # 按完成顺序获取结果\n",
    "#     for future in as_completed(futures):\n",
    "#         res = future.result()\n",
    "#         logger.info(f'future.result: {len(res)}')\n",
    "#         results.append(res)\n",
    "    \n",
    "# llm_country_describe = [''.join(x) for r in results for x in r]\n",
    "# end = time.time()\n",
    "# logger.info(f'country_normal_mapping takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "917dac4a-0b71-46c1-9060-0706a78007ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(llm_country_describe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7e8c0ce-e8b6-4ab9-9ef6-8bc205ef0a3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 单线程\n",
    "# llm_country_describe=get_llm_country_normal2(all_data_country_list, batch=20, step=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0062d9ab-c528-4918-ae3f-841ff98f42c3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "llm_country_describe = model_result['llm_country_describe']\n",
    "# llm_country_describe"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bcc51a4-c2e8-41aa-b170-34d454180dc7",
   "metadata": {},
   "source": [
    "## 1.3 LLM结果解析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ba2408f-0725-48fc-9151-4887dd8bd174",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_json_text(text):\n",
    "    # 替换常见的非法转义字符\n",
    "    text = re.sub(r'\\\\(?![\\\"\\\\/bfnrt])', r'\\\\\\\\', text)\n",
    "    return text\n",
    "\n",
    "with open('param_files/ctry_dict_20250810.log', 'r', encoding='utf-8') as f:\n",
    "    raw = f.read()\n",
    "\n",
    "cleaned = clean_json_text(raw)\n",
    "\n",
    "try:\n",
    "    tmp = json.loads(cleaned)\n",
    "    print(\"成功解析 JSON，共 {} 条记录\".format(len(tmp)))\n",
    "except json.JSONDecodeError as e:\n",
    "    print(\"解析失败:\", e)\n",
    "\n",
    "tmp1 = pd.DataFrame(tmp).drop_duplicates()\n",
    "tmp1.reset_index(drop=True, inplace=True)\n",
    "llm_country_mapping = tmp1.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6202f7e9-d3f9-4dc3-8ba2-50c1ab822415",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "country_mapping = merge_country_info(all_data_country_list, llm_country_mapping, country_code)\n",
    "country_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c553d66c-7d26-40cc-8bd1-62813b909759",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "559a7ac1-7d5c-4abc-80f3-44cf8eaca2f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8ef28630-b9a0-452b-a1b1-3881d8ec4ae5",
   "metadata": {},
   "source": [
    "## 1.4 人工添加国家对应关系"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "353a7262-ea15-4a28-b550-dd7c92cfc809",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 'BENÃ\\xadN': {'中文国名': '贝宁', '英文国名': 'Benin', '国别编码': 'BEN'},\n",
    "# # 'ETIOPÃ\\xadA': {'中文国名': '埃塞俄比亚', '英文国名': 'Ethiopia', '国别编码': 'ETH'},\n",
    "# # 'HUNGRÃ\\xadA': {'中文国名': '匈牙利', '英文国名': 'Hungary', '国别编码': 'HUN'},\n",
    "# # 'LÃ\\xadBANO': {'中文国名': '黎巴嫩', '英文国名': 'Lebanon', '国别编码': 'LBN'},\n",
    "# # 'SANTA LUCÃ\\xadA': {'中文国名': '圣卢西亚', '英文国名': 'Saint Lucia', '国别编码': 'LCA'},\n",
    "# # 'SANTO TOMÃ© Y PRÃ\\xadNCIPE': {'中文国名': '圣多美和普林西比', '英文国名': 'Sao Tome and Principe', '国别编码': 'STP'},\n",
    "# # 'SYRIAN ARAB REPUBLIC\\xa0(SYRIA)': {'中文国名': '叙利亚', '英文国名': 'Syrian Arab Republic', '国别编码': 'SYR'},\n",
    "# # 'TANZANIA\\xa0*, UNITED REPUBLIC OF': {'中文国名': '坦桑尼亚', '英文国名': 'Tanzania, United Republic of', '国别编码': 'TZA'},\n",
    "# # 'TURQUÃ\\xadA': {'中文国名': '土耳其', '英文国名': 'Turkey', '国别编码': 'TUR'},\n",
    "# # 'PAÃ\\xadS AFRICANO NO DETERMINADO': {'中文国名': '', '英文国名': '', '国别编码': ''},\n",
    "# # 'PAÃ\\xadS ASIÃ¡TICO NO DETERMINADO': {'中文国名': '', '英文国名': '', '国别编码': ''},\n",
    "# # 'PAÃ\\xadS EUROPEO NO DETERMINADO': {'中文国名': '', '英文国名': '', '国别编码': ''},\n",
    "# # 'RÃ\\xadOS NACIONALES ARGENTINOS DE NAVEGACIÃ³N INTERNACIONAL': {'中文国名': '', '英文国名': '', '国别编码': ''},\n",
    "# # 'TVA AUSTRALIA (OCEANÃ\\xadA)': {'中文国名': '', '英文国名': '', '国别编码': ''},\n",
    "# # 'ZONA FRANCA RÃ\\xadO GALLEGOS': {'中文国名': '', '英文国名': '', '国别编码': ''},\n",
    "# # 'ZONA FRANCA RÃ\\xadO NEGRO': {'中文国名': '', '英文国名': '', '国别编码': ''},\n",
    "\n",
    "# raw = Path(file_path).read_text(encoding='utf-8')\n",
    "# # raw = Path(file_path).read_text(encoding='latin1').encode('latin1').decode('utf-8')\n",
    "\n",
    "# clean = raw.replace(\"'\", '\"')\n",
    "\n",
    "# country_mapping_new = json.loads(clean)\n",
    "# print(country_mapping_new['ZANZIBAR'])          # {'中文国名': '坦桑尼亚', '英文国名': 'Tanzania', '国别编码': 'TZA'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eced4f3d-6c2e-4caa-af27-9ac793f68f39",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = 'param_files/country_mapping_dict.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ddc1dc3-241c-41fe-a3c8-6d0ac2866164",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "raw = Path(file_path).read_text(encoding='utf-8')\n",
    "# raw = Path(file_path).read_text(encoding='latin1').encode('latin1').decode('utf-8')\n",
    "\n",
    "clean = raw.replace(\"'\", '\"')\n",
    "\n",
    "try:\n",
    "    country_mapping_new = ast.literal_eval(clean)\n",
    "except (SyntaxError, ValueError) as e:\n",
    "    print(f\"解析错误: {e}\")\n",
    "    \n",
    "print(country_mapping_new['ZANZIBAR'])          # {'中文国名': '坦桑尼亚', '英文国名': 'Tanzania', '国别编码': 'TZA'}\n",
    "# country_mapping_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0df8e7d0-7b1a-41f1-82d3-1627e9c0fa08",
   "metadata": {},
   "outputs": [],
   "source": [
    "# country_mapping_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d968f0f-8da8-450c-9bbb-c94604978693",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # 使用函数读取文件\n",
    "# country_mapping_new = read_and_fix_encoded_file(file_path)\n",
    "\n",
    "# # if country_mapping_new:\n",
    "# #     print(\"成功读取字典:\")\n",
    "# #     for key, value in country_mapping_new.items():\n",
    "# #         print(f\"{repr(key)}: {value}\")  # 使用 repr 显示原始表示\n",
    "# # else:\n",
    "# #     print(\"无法解析文件内容\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a879660-58d7-45d1-9493-e9a80a4d5d49",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# country_mapping_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1ea5f81-81fa-45e2-a054-a28a65a8e969",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(len(country_mapping))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7104ecb2-b27f-4219-a8bf-05cc7d56fe03",
   "metadata": {},
   "outputs": [],
   "source": [
    "country_mapping.update(country_mapping_new)    # 把 d2 并进来"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34a4800f-f27e-4992-935a-fcbbb44502fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(country_mapping))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9712326-3063-4bf3-bbef-ced5cee29daa",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_result['country_mapping'] = country_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "393ddce7-1ec5-444e-97d1-55852f1e89d2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35cf7243-ede8-4b9a-9a1a-db735531f704",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for x, sub in country_mapping.items():\n",
    "#     code = sub.get('国别编码')\n",
    "#     if is_nan(code):\n",
    "#         print(x, code)  # 只打印键和缺失的编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "013f8063-f026-4b52-8c46-7e21f06a0ae0",
   "metadata": {},
   "outputs": [],
   "source": [
    "missing = {\n",
    "    k: v.get('国别编码')\n",
    "    for k, v in country_mapping.items()\n",
    "    if is_nan(v.get('国别编码'))\n",
    "}\n",
    "# print(missing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0af681a7-0c01-431a-877d-66324202b962",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(missing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74d8bf7f-261e-46a2-9fc2-767debd30994",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd4a7ad1-019e-4399-88c3-a8a6707c5430",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# for x in country_mapping.keys():\n",
    "#     if is_nan(country_mapping[x].get('国别编码')):\n",
    "#         # country_mapping[x]['国别编码']=''\n",
    "#         print(country_mapping[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "334445d4-e872-4c3f-8da8-852534c73dba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# country_mapping['BENÃ\\xadN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5822520c-b77d-49ea-b695-58d6f719d81a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# country_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "139fde04-9e6e-4bc8-8a1c-abd2458c64b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(model_result, open('result/model_rslt_05_20250901.pickle', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5e4532a-df87-4ba2-a981-a650ea23e3d7",
   "metadata": {},
   "source": [
    "# 2 货币单位校正"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b8437f4-546c-4280-ae04-9f78cccb282f",
   "metadata": {},
   "source": [
    "## 2.1 利用世界银行python包wbgapi获取全世界年度货币汇率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05c63f5b-3bc8-4a86-acff-ceb950ad3bf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "currency_df = get_currency_exchange_rate(2023,2025)\n",
    "model_result['currency_df'] = currency_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f6ed661-def9-40c8-b15b-7cfb920a262c",
   "metadata": {},
   "outputs": [],
   "source": [
    "currency_df.loc[currency_df['id']=='ARG',:]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8dcee7a-726b-47c7-80d1-115e3072f94f",
   "metadata": {},
   "source": [
    "## 2.2 获取各国货币编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c655792-d235-44db-9362-135b344b0295",
   "metadata": {},
   "outputs": [],
   "source": [
    "country_list = ['印度', '英国', '越南', '阿根廷', '埃塞俄比亚', '巴基斯坦', '俄罗斯陆运', '厄瓜多尔'\n",
    "                , '菲律宾', '哥伦比亚', '哥斯达黎加', '哈萨克斯坦', '加纳', '喀麦隆', '科特迪瓦', '肯尼亚'\n",
    "                , '莱索托', '马拉维', '美国', '孟加拉', '秘鲁', '秘鲁-海运', '秘鲁-空运', '墨西哥'\n",
    "                , '纳米比亚', '尼日利亚', '斯里兰卡', '坦桑尼亚', '乌干达', '乌克兰', '乌拉圭', '乌兹别克斯坦'\n",
    "                , '智利', '巴拉圭', '巴拿马', '博兹瓦纳']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "709d4851-0d7d-49e2-80c5-5dba0e492f06",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from_path = Path('dataset/data_20250724/20251t3_csv_correct2')\n",
    "currency_cols = ['cif_currency', 'fob_currency']\n",
    "# currency_list = get_currency_list(country_list, from_path, currency_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5af065dc-8d6f-444b-af86-7efef418be64",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 进口跑批\n",
    "start = time.time()\n",
    "\n",
    "params_list = [([ctry], from_path, currency_cols) for ctry in country_list]\n",
    "currency_list = []\n",
    "with ThreadPoolExecutor(max_workers=16) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: get_currency_list(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        res = future.result()\n",
    "        logger.info(len(res))\n",
    "        currency_list+=res\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'get_currency_list takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e03c453-d207-4298-a4d9-9a4acf069fcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "currency_list = list(set(currency_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5a33d2e-dc18-44e9-a834-6b1e72e5e7d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(currency_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16367fee-bdf1-48e9-a4c0-9e6152e7cdb8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 将currency_list粘贴给大模型，“以上是哪国货币，其对应的国家英文名称和三位ISO国家编码是多少，请给出对应列表”\n",
    "currency_list_df = pd.read_csv('param_files/货币编码_20250807.csv',encoding='utf-8',header=0)\n",
    "currency_list_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73efb50f-f974-4644-acdb-03130c300cb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "currency_on_usd_df = pd.merge(currency_list_df.loc[:, ['货币代码', 'ISO国家编码']], currency_df.loc[:, ['id', 'exchange_rate_on_usd']]\n",
    "                              , left_on='ISO国家编码', right_on='id', how='left')\n",
    "# currency_on_usd_df\n",
    "currency_on_usd_df.loc[currency_on_usd_df['货币代码']=='TWD', 'exchange_rate_on_usd']=29.3920\n",
    "currency_on_usd_df.loc[currency_on_usd_df['货币代码']=='EUR', 'exchange_rate_on_usd']=0.8574\n",
    "# currency_on_usd_df.loc[currency_on_usd_df['货币代码']=='IND', 'exchange_rate_on_usd']=currency_on_usd_df.loc[currency_on_usd_df['货币代码']=='INR', 'exchange_rate_on_usd'].values\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d98f5cb-b2bf-474a-b363-7c29b1baf533",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "currency_on_usd_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f6c57d1-d6b0-4007-acdd-e29e96ab61bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "currency_on_usd=dict()\n",
    "for x in currency_on_usd_df.values:\n",
    "    currency_on_usd[x[0]]=round(x[3],4)\n",
    "\n",
    "for x in currency_df.loc[:, ['id', 'exchange_rate_on_usd']].values:\n",
    "    currency_on_usd[x[0]]=x[1]\n",
    "\n",
    "print(len(currency_on_usd))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b01c629-58a1-4fe5-9407-5ca08286cf6c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "currency_on_usd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35e22d9c-e4c7-4ae1-a7c8-7a3eb20eed30",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_result['currency_on_usd'] = currency_on_usd\n",
    "pickle.dump(model_result, open('result/model_rslt_05_20250901.pickle', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76fd42f8-3d1b-4159-83d4-19f29ab2fec5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "03e2a073-1e13-4b59-acc8-370cf2304545",
   "metadata": {},
   "source": [
    "# 3 获取HS码映射关系"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51a57e6c-f029-45f5-a7b1-5ebae9b45563",
   "metadata": {},
   "outputs": [],
   "source": [
    "# hscode\tstring\ths编码\t\tHS编码和产品描述至少一个不为空。\n",
    "# hscodedescription\tstring\ths编码描述"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82f164fa-7133-4a3a-956c-c89b5d968ace",
   "metadata": {},
   "outputs": [],
   "source": [
    "from_path = Path('dataset/data_20250724/20251t3_csv_correct1')\n",
    "to_path = Path('result/20251t3_hs_mapping')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "947e03a8-8e84-4198-ba79-b0f5294860a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "country_list = ['印度', '英国', '越南', '阿根廷', '埃塞俄比亚', '巴基斯坦', '俄罗斯陆运', '厄瓜多尔'\n",
    "                , '菲律宾', '哥伦比亚', '哥斯达黎加', '哈萨克斯坦', '加纳', '喀麦隆', '科特迪瓦', '肯尼亚'\n",
    "                , '莱索托', '马拉维', '美国', '孟加拉', '秘鲁', '秘鲁-海运', '秘鲁-空运', '墨西哥'\n",
    "                , '纳米比亚', '尼日利亚', '斯里兰卡', '坦桑尼亚', '乌干达', '乌克兰', '乌拉圭', '乌兹别克斯坦'\n",
    "                , '智利', '巴拉圭', '巴拿马', '博兹瓦纳']\n",
    "# country_list = ['尼日利亚', '博兹瓦纳']\n",
    "# country_list = ['阿根廷']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49beb2ce-6bd5-41a4-a0ea-cec9f83421cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "params_list = [(from_path, to_path, [ctry]) for ctry in country_list]\n",
    "\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: get_hs_mapping(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        logger.info(future.result().shape)\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'get_type_enumvalue takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d563d083-874e-4de2-93fd-0c97cc2aee4f",
   "metadata": {},
   "source": [
    "# 4 获取成交方式、运输方式、付款方式、"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "61705263-9f54-47e3-9d57-8c82b31c4574",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transportterm    运输方式\n",
    "# tradeterm    成交方式\n",
    "# paymentterm    付款方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1b809985-6a97-4b10-93c8-4a02cebc3bb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from_path = Path('dataset/data_20250724/20251t3_csv_correct1')\n",
    "# to_path = Path('result')\n",
    "\n",
    "from_path = Path('dataset/data_20250724/20254t6_csv_correct1')\n",
    "to_path = Path('result')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cc3fd48c-38fe-4b0c-8b14-4e10748fbd9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "country_list = ['印度', '英国', '越南', '阿根廷', '埃塞俄比亚', '巴基斯坦', '俄罗斯陆运', '厄瓜多尔'\n",
    "                , '菲律宾', '哥伦比亚', '哥斯达黎加', '哈萨克斯坦', '加纳', '喀麦隆', '科特迪瓦', '肯尼亚'\n",
    "                , '莱索托', '马拉维', '美国', '孟加拉', '秘鲁', '秘鲁-海运', '秘鲁-空运', '墨西哥'\n",
    "                , '纳米比亚', '尼日利亚', '斯里兰卡', '坦桑尼亚', '乌干达', '乌克兰', '乌拉圭', '乌兹别克斯坦'\n",
    "                , '智利', '巴拉圭', '巴拿马', '博兹瓦纳']\n",
    "# country_list = ['阿根廷']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2151d3d8-ab31-4e68-96d8-41ba4d1213d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# rslt = get_type_enumvalue(from_path, to_path, country_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b89bde73-a87c-4c66-8b00-fcad354dcec0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-10 05:20:28,625 - File Process - INFO - 英国\n",
      "2025-09-10 05:20:28,628 - File Process - INFO - 巴基斯坦\n",
      "2025-09-10 05:20:28,628 - File Process - INFO - 越南\n",
      "2025-09-10 05:20:28,629 - File Process - INFO - 菲律宾\n",
      "2025-09-10 05:20:28,630 - File Process - INFO - 印度\n",
      "2025-09-10 05:20:28,631 - File Process - INFO - 阿根廷\n",
      "2025-09-10 05:20:28,632 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,632 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,634 - File Process - INFO - 哥伦比亚\n",
      "2025-09-10 05:20:28,634 - File Process - INFO - 埃塞俄比亚\n",
      "2025-09-10 05:20:28,635 - File Process - INFO - 俄罗斯陆运\n",
      "2025-09-10 05:20:28,635 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,637 - File Process - INFO - 厄瓜多尔\n",
      "2025-09-10 05:20:28,638 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,640 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,641 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,641 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,649 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,641 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,646 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,650 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,650 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,647 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,648 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,648 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,648 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,648 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,653 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,648 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,649 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,654 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,651 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,652 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,655 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,656 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,652 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,657 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,653 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,658 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,659 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,671 - File Process - INFO - 哥斯达黎加\n",
      "2025-09-10 05:20:28,673 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,674 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,675 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,676 - File Process - INFO - 加纳\n",
      "2025-09-10 05:20:28,678 - File Process - INFO - 哈萨克斯坦\n",
      "2025-09-10 05:20:28,680 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,681 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,681 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,682 - File Process - INFO - 马拉维\n",
      "2025-09-10 05:20:28,682 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,683 - File Process - INFO - 美国\n",
      "2025-09-10 05:20:28,685 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,686 - File Process - INFO - 科特迪瓦\n",
      "2025-09-10 05:20:28,688 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,686 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,687 - File Process - INFO - 莱索托\n",
      "2025-09-10 05:20:28,689 - File Process - INFO - 肯尼亚\n",
      "2025-09-10 05:20:28,689 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,689 - File Process - INFO - 喀麦隆\n",
      "2025-09-10 05:20:28,691 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,691 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,698 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,693 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,695 - File Process - INFO - 孟加拉\n",
      "2025-09-10 05:20:28,695 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,695 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,698 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,702 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,698 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,699 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,703 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,701 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,704 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,701 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,705 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,702 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,706 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,705 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,707 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,711 - File Process - INFO - 秘鲁\n",
      "2025-09-10 05:20:28,714 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,715 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,716 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,716 - File Process - INFO - 秘鲁-海运\n",
      "2025-09-10 05:20:28,719 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,720 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,720 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,723 - File Process - INFO - 秘鲁-空运\n",
      "2025-09-10 05:20:28,725 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,726 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,727 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,730 - File Process - INFO - 墨西哥\n",
      "2025-09-10 05:20:28,733 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,733 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,734 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,735 - File Process - INFO - 纳米比亚\n",
      "2025-09-10 05:20:28,736 - File Process - INFO - 尼日利亚\n",
      "2025-09-10 05:20:28,740 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,741 - File Process - INFO - 斯里兰卡\n",
      "2025-09-10 05:20:28,741 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,741 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,744 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,742 - File Process - INFO - 乌克兰\n",
      "2025-09-10 05:20:28,744 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,744 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,747 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,745 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202504.csv\n",
      "2025-09-10 05:20:28,746 - File Process - INFO - 乌干达\n",
      "2025-09-10 05:20:28,747 - File Process - INFO - 坦桑尼亚\n",
      "2025-09-10 05:20:28,748 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,751 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,752 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,751 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202504-IMP-RAW.csv\n",
      "2025-09-10 05:20:28,752 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,754 - File Process - INFO - 乌拉圭\n",
      "2025-09-10 05:20:28,755 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,757 - File Process - INFO - 乌兹别克斯坦\n",
      "2025-09-10 05:20:28,759 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,760 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,761 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,761 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,761 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,762 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,764 - File Process - INFO - 智利\n",
      "2025-09-10 05:20:28,765 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202504-IMP.csv\n",
      "2025-09-10 05:20:28,767 - File Process - INFO - 巴拉圭\n",
      "2025-09-10 05:20:28,769 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,770 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,770 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,771 - File Process - INFO - 巴拿马\n",
      "2025-09-10 05:20:28,773 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,773 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,774 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:28,775 - File Process - INFO - 博兹瓦纳\n",
      "2025-09-10 05:20:28,777 - File Process - INFO - type_all_unique shape: (0, 4)\n",
      "2025-09-10 05:20:28,777 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:28,778 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:31,672 - File Process - INFO - table shape: (387273, 3)\n",
      "2025-09-10 05:20:31,672 - File Process - INFO - hs unique shape: (10, 3)\n",
      "2025-09-10 05:20:31,673 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202506-IMP-RAW.csv\n",
      "2025-09-10 05:20:32,647 - File Process - INFO - table shape: (390891, 3)\n",
      "2025-09-10 05:20:32,648 - File Process - INFO - hs unique shape: (191, 3)\n",
      "2025-09-10 05:20:32,648 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202505-IMP.csv\n",
      "2025-09-10 05:20:34,844 - File Process - INFO - table shape: (344016, 3)\n",
      "2025-09-10 05:20:34,845 - File Process - INFO - hs unique shape: (9, 3)\n",
      "2025-09-10 05:20:34,845 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/IMPORT/202505-IMP-RAW.csv\n",
      "2025-09-10 05:20:36,821 - File Process - INFO - table shape: (404234, 3)\n",
      "2025-09-10 05:20:36,822 - File Process - INFO - hs unique shape: (205, 3)\n",
      "2025-09-10 05:20:36,823 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/智利/IMPORT/202506-IMP.csv\n",
      "2025-09-10 05:20:37,619 - File Process - INFO - table shape: (310571, 3)\n",
      "2025-09-10 05:20:37,620 - File Process - INFO - hs unique shape: (8, 3)\n",
      "2025-09-10 05:20:37,620 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-TRANS-RAW.csv\n",
      "2025-09-10 05:20:38,812 - File Process - INFO - table shape: (128934, 3)\n",
      "2025-09-10 05:20:38,813 - File Process - INFO - hs unique shape: (6, 3)\n",
      "2025-09-10 05:20:38,814 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202504-EXP-RAW.csv\n",
      "2025-09-10 05:20:38,847 - File Process - INFO - table shape: (6534, 3)\n",
      "2025-09-10 05:20:38,847 - File Process - INFO - hs unique shape: (8, 3)\n",
      "2025-09-10 05:20:38,848 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-EXP-RAW.csv\n",
      "2025-09-10 05:20:38,953 - File Process - INFO - table shape: (5331, 3)\n",
      "2025-09-10 05:20:38,953 - File Process - INFO - hs unique shape: (6, 3)\n",
      "2025-09-10 05:20:38,954 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-TRANS-RAW.csv\n",
      "2025-09-10 05:20:41,501 - File Process - INFO - table shape: (138793, 3)\n",
      "2025-09-10 05:20:41,505 - File Process - INFO - hs unique shape: (6, 3)\n",
      "2025-09-10 05:20:41,510 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202506-EXP-RAW.csv\n",
      "2025-09-10 05:20:41,518 - File Process - INFO - table shape: (739143, 3)\n",
      "2025-09-10 05:20:41,519 - File Process - INFO - hs unique shape: (16, 3)\n",
      "2025-09-10 05:20:41,519 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202506.csv\n",
      "2025-09-10 05:20:41,543 - File Process - INFO - table shape: (5108, 3)\n",
      "2025-09-10 05:20:41,543 - File Process - INFO - hs unique shape: (6, 3)\n",
      "2025-09-10 05:20:41,544 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/坦桑尼亚/EXPORT/202505-TRANS-RAW.csv\n",
      "2025-09-10 05:20:42,411 - File Process - INFO - table shape: (400856, 3)\n",
      "2025-09-10 05:20:42,412 - File Process - INFO - hs unique shape: (186, 3)\n",
      "2025-09-10 05:20:42,413 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202506-EXP.csv\n",
      "2025-09-10 05:20:43,087 - File Process - INFO - table shape: (125615, 3)\n",
      "2025-09-10 05:20:43,088 - File Process - INFO - hs unique shape: (6, 3)\n",
      "2025-09-10 05:20:43,091 - File Process - INFO - type_all_unique shape: (10, 4)\n",
      "2025-09-10 05:20:43,091 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:43,092 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:43,226 - File Process - INFO - table shape: (87003, 3)\n",
      "2025-09-10 05:20:43,226 - File Process - INFO - hs unique shape: (51, 3)\n",
      "2025-09-10 05:20:43,227 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202505-EXP.csv\n",
      "2025-09-10 05:20:43,744 - File Process - INFO - table shape: (91874, 3)\n",
      "2025-09-10 05:20:43,744 - File Process - INFO - hs unique shape: (49, 3)\n",
      "2025-09-10 05:20:43,745 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/智利/EXPORT/202504-EXP.csv\n",
      "2025-09-10 05:20:44,360 - File Process - INFO - table shape: (112769, 3)\n",
      "2025-09-10 05:20:44,361 - File Process - INFO - hs unique shape: (51, 3)\n",
      "2025-09-10 05:20:44,363 - File Process - INFO - type_all_unique shape: (415, 4)\n",
      "2025-09-10 05:20:44,364 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:44,365 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:47,595 - File Process - INFO - table shape: (744475, 3)\n",
      "2025-09-10 05:20:47,595 - File Process - INFO - hs unique shape: (15, 3)\n",
      "2025-09-10 05:20:47,596 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/乌克兰/IMPORT/UA_IMPORT_202505.csv\n",
      "2025-09-10 05:20:51,883 - File Process - INFO - table shape: (755738, 3)\n",
      "2025-09-10 05:20:51,883 - File Process - INFO - hs unique shape: (15, 3)\n",
      "2025-09-10 05:20:51,884 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202506.csv\n",
      "2025-09-10 05:20:52,705 - File Process - INFO - table shape: (134581, 3)\n",
      "2025-09-10 05:20:52,706 - File Process - INFO - hs unique shape: (15, 3)\n",
      "2025-09-10 05:20:52,706 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202504.csv\n",
      "2025-09-10 05:20:53,530 - File Process - INFO - table shape: (135921, 3)\n",
      "2025-09-10 05:20:53,531 - File Process - INFO - hs unique shape: (15, 3)\n",
      "2025-09-10 05:20:53,531 - File Process - INFO - data from: dataset/data_20250724/20254t6_csv_correct1/乌克兰/EXPORT/UA_EXPORT_202505.csv\n",
      "2025-09-10 05:20:54,420 - File Process - INFO - table shape: (147189, 3)\n",
      "2025-09-10 05:20:54,420 - File Process - INFO - hs unique shape: (15, 3)\n",
      "2025-09-10 05:20:54,423 - File Process - INFO - type_all_unique shape: (16, 4)\n",
      "2025-09-10 05:20:54,423 - File Process - INFO - new data save: result/type_unique.csv\n",
      "2025-09-10 05:20:54,424 - Data Goverance - INFO - (1, 4)\n",
      "2025-09-10 05:20:54,428 - Data Goverance - INFO - get_type_enumvalue takes 0.430389948685964 minutes\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "params_list = [(from_path, to_path, [ctry]) for ctry in country_list]\n",
    "\n",
    "rslt=pd.DataFrame()\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: get_type_enumvalue(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        logger.info(future.result().shape)\n",
    "        rslt = pd.concat([rslt, future.result()], axis=0, ignore_index=True)\n",
    "        \n",
    "\n",
    "# rslt = pd.concat([future.result() for future in futures], axis=0, ignore_index=True)\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'get_type_enumvalue takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a20e5459-aea3-4cd7-a6ce-986233851238",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>transportterm</th>\n",
       "      <th>tradeterm</th>\n",
       "      <th>paymentterm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>英国</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>巴基斯坦</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>越南</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>菲律宾</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>印度</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>阿根廷</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>埃塞俄比亚</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>哥伦比亚</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>俄罗斯陆运</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>厄瓜多尔</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>哥斯达黎加</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>加纳</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>哈萨克斯坦</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>马拉维</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>美国</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>科特迪瓦</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>莱索托</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>肯尼亚</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>喀麦隆</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>孟加拉</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>秘鲁</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>秘鲁-海运</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>秘鲁-空运</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>墨西哥</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>纳米比亚</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>尼日利亚</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>斯里兰卡</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>乌干达</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>乌拉圭</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>乌兹别克斯坦</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>巴拉圭</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>巴拿马</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>博兹瓦纳</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>坦桑尼亚</td>\n",
       "      <td>[UNKNOWN, AIR, TRANSPORT ON FIXED INSTALLATION...</td>\n",
       "      <td>[nan]</td>\n",
       "      <td>[nan]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>智利</td>\n",
       "      <td>[AIR, MAIL, PIPELINE, nan, ROAD, OTHERS, SEA]</td>\n",
       "      <td>[CFR, FAS, DAP, FCA, DAT, CIF, EXW, OTRO, SIN ...</td>\n",
       "      <td>[ANTICIPADO 20% - CREDITO 80% HASTA 1 AÑO, ANT...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>乌克兰</td>\n",
       "      <td>[OTHERS]</td>\n",
       "      <td>[DAF, CFR, DAP, FCA, DAT, FAS, CIF, DPU, EXW, ...</td>\n",
       "      <td>[nan]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   country                                      transportterm  \\\n",
       "0       英国                                                 []   \n",
       "1     巴基斯坦                                                 []   \n",
       "2       越南                                                 []   \n",
       "3      菲律宾                                                 []   \n",
       "4       印度                                                 []   \n",
       "5      阿根廷                                                 []   \n",
       "6    埃塞俄比亚                                                 []   \n",
       "7     哥伦比亚                                                 []   \n",
       "8    俄罗斯陆运                                                 []   \n",
       "9     厄瓜多尔                                                 []   \n",
       "10   哥斯达黎加                                                 []   \n",
       "11      加纳                                                 []   \n",
       "12   哈萨克斯坦                                                 []   \n",
       "13     马拉维                                                 []   \n",
       "14      美国                                                 []   \n",
       "15    科特迪瓦                                                 []   \n",
       "16     莱索托                                                 []   \n",
       "17     肯尼亚                                                 []   \n",
       "18     喀麦隆                                                 []   \n",
       "19     孟加拉                                                 []   \n",
       "20      秘鲁                                                 []   \n",
       "21   秘鲁-海运                                                 []   \n",
       "22   秘鲁-空运                                                 []   \n",
       "23     墨西哥                                                 []   \n",
       "24    纳米比亚                                                 []   \n",
       "25    尼日利亚                                                 []   \n",
       "26    斯里兰卡                                                 []   \n",
       "27     乌干达                                                 []   \n",
       "28     乌拉圭                                                 []   \n",
       "29  乌兹别克斯坦                                                 []   \n",
       "30     巴拉圭                                                 []   \n",
       "31     巴拿马                                                 []   \n",
       "32    博兹瓦纳                                                 []   \n",
       "33    坦桑尼亚  [UNKNOWN, AIR, TRANSPORT ON FIXED INSTALLATION...   \n",
       "34      智利      [AIR, MAIL, PIPELINE, nan, ROAD, OTHERS, SEA]   \n",
       "35     乌克兰                                           [OTHERS]   \n",
       "\n",
       "                                            tradeterm  \\\n",
       "0                                                  []   \n",
       "1                                                  []   \n",
       "2                                                  []   \n",
       "3                                                  []   \n",
       "4                                                  []   \n",
       "5                                                  []   \n",
       "6                                                  []   \n",
       "7                                                  []   \n",
       "8                                                  []   \n",
       "9                                                  []   \n",
       "10                                                 []   \n",
       "11                                                 []   \n",
       "12                                                 []   \n",
       "13                                                 []   \n",
       "14                                                 []   \n",
       "15                                                 []   \n",
       "16                                                 []   \n",
       "17                                                 []   \n",
       "18                                                 []   \n",
       "19                                                 []   \n",
       "20                                                 []   \n",
       "21                                                 []   \n",
       "22                                                 []   \n",
       "23                                                 []   \n",
       "24                                                 []   \n",
       "25                                                 []   \n",
       "26                                                 []   \n",
       "27                                                 []   \n",
       "28                                                 []   \n",
       "29                                                 []   \n",
       "30                                                 []   \n",
       "31                                                 []   \n",
       "32                                                 []   \n",
       "33                                              [nan]   \n",
       "34  [CFR, FAS, DAP, FCA, DAT, CIF, EXW, OTRO, SIN ...   \n",
       "35  [DAF, CFR, DAP, FCA, DAT, FAS, CIF, DPU, EXW, ...   \n",
       "\n",
       "                                          paymentterm  \n",
       "0                                                  []  \n",
       "1                                                  []  \n",
       "2                                                  []  \n",
       "3                                                  []  \n",
       "4                                                  []  \n",
       "5                                                  []  \n",
       "6                                                  []  \n",
       "7                                                  []  \n",
       "8                                                  []  \n",
       "9                                                  []  \n",
       "10                                                 []  \n",
       "11                                                 []  \n",
       "12                                                 []  \n",
       "13                                                 []  \n",
       "14                                                 []  \n",
       "15                                                 []  \n",
       "16                                                 []  \n",
       "17                                                 []  \n",
       "18                                                 []  \n",
       "19                                                 []  \n",
       "20                                                 []  \n",
       "21                                                 []  \n",
       "22                                                 []  \n",
       "23                                                 []  \n",
       "24                                                 []  \n",
       "25                                                 []  \n",
       "26                                                 []  \n",
       "27                                                 []  \n",
       "28                                                 []  \n",
       "29                                                 []  \n",
       "30                                                 []  \n",
       "31                                                 []  \n",
       "32                                                 []  \n",
       "33                                              [nan]  \n",
       "34  [ANTICIPADO 20% - CREDITO 80% HASTA 1 AÑO, ANT...  \n",
       "35                                              [nan]  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rslt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "13e1f097-5902-48bb-a27a-c3ab35669ce2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['UNKNOWN',\n",
       " 'TRANSPORT ON FIXED INSTALLATION',\n",
       " 'AIR',\n",
       " 'nan',\n",
       " 'MAIL',\n",
       " 'RAILWAY',\n",
       " 'INLAND WATERWAYS TRANSPORT',\n",
       " 'PIPELINE',\n",
       " 'SEA TRANSPORT',\n",
       " 'ROAD',\n",
       " 'MULTIMODAL TRANSPORT',\n",
       " 'OTHERS',\n",
       " 'SEA',\n",
       " 'POSTAL TRANSPORT']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 运输方式\n",
    "transportterm_list = list(set([y.upper() if not is_nan(y) else 'nan' for x in rslt['transportterm'].values for y in x]))\n",
    "print(len(transportterm_list))\n",
    "transportterm_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5486b591-b9eb-4aec-b681-ab75eb24fe1e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['CFR',\n",
       " 'DAP',\n",
       " 'DAT',\n",
       " 'CIF',\n",
       " 'OTRO',\n",
       " 'EXW',\n",
       " 'CIP',\n",
       " 'DDU',\n",
       " 'DDP',\n",
       " 'DAF',\n",
       " 'FAS',\n",
       " 'FCA',\n",
       " 'nan',\n",
       " 'CPT',\n",
       " 'FOB',\n",
       " 'DPU',\n",
       " 'SIN CLAUSULA',\n",
       " 'CYS',\n",
       " '000']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 成交方式\n",
    "tradeterm_list = list(set([y.upper() if not is_nan(y) else 'nan' for x in rslt['tradeterm'].values for y in x]))\n",
    "print(len(tradeterm_list))\n",
    "tradeterm_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9d6979fd-8b12-4fba-bba7-cc5d63ac057a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['ANTICIPADO 20% - CREDITO 80% HASTA 1 AÑO',\n",
       " 'ANTICIPADO 05% - CREDITO 95%',\n",
       " 'ANTICIPADO 40% - CREDITO 60% HASTA 1 AÑO',\n",
       " 'COBRANZA MÁS DE 1 AÑO',\n",
       " 'CREDITO BANCOS Y FINANCIEROS M?S DE 1 A?O',\n",
       " 'ANTICIPADO 10% - COBRANZA 90% HASTA 1 A?O',\n",
       " 'ANTICIPADO 50% - CREDITO 50% HASTA 1 AÑO',\n",
       " 'ANTICIPADO 15% - COBRANZA 85%',\n",
       " 'ACREDITIVO HASTA 1 AÑO',\n",
       " 'ANTICIPADO 30% - CREDITO 70% HASTA 1 A?O',\n",
       " 'SIN PAGO',\n",
       " 'COBRANZA HASTA 1 AÑO',\n",
       " 'ANTICIPADO 40% - CREDITO 60% HASTA 1 A?O',\n",
       " 'ACREDITIVO HASTA 1 A?O',\n",
       " 'ANTICIPADO 30% - CREDITO 70%',\n",
       " 'ANTICIPADO (+ 50%) - CREDITO (-50%) HASTA 1 AÑO',\n",
       " 'COBRANZA M?S DE 1 A?O',\n",
       " 'nan',\n",
       " 'ANTICIPADO 50% - CREDITO 50% HASTA 1 A?O',\n",
       " 'ANTICIPADO (+ 50%) - CREDITO (-50%) HASTA 1 A?O',\n",
       " 'ANTICIPADO 30% - CREDITO 70% HASTA 1 AÑO',\n",
       " 'ANTICIPADO 10% - COBRANZA 90% HASTA 1 AÑO',\n",
       " 'ANTICIPADO 20% - CREDITO 80% HASTA 1 A?O',\n",
       " 'PAGO ANTICIPADO A FECHA DE EMBARQUE',\n",
       " 'COBRANZA HASTA 1 A?O',\n",
       " 'SIN PAGO COBERTURA',\n",
       " 'ANTICIPADO 05% - COBRANZA 95%']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 付款方式\n",
    "paymentterm_list = list(set([y.upper() if not is_nan(y) else 'nan' for x in rslt['paymentterm'].values for y in x]))\n",
    "print(len(paymentterm_list))\n",
    "paymentterm_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea62745b-3881-4691-b6eb-fc65fb0fa90d",
   "metadata": {},
   "outputs": [],
   "source": [
    "lst1 = ['VESSEL, CONTAINERIZED', 'L', 'NOT DECLARED', 'PIPELINE', '50', 'AERIAL', 'AIR'\n",
    "        , 'LAND - OTHER', 'OLEODUCTOS', 'АВТОТРАНСПОРТ СВОЇМ ХОДОМ ЯК ТОВАР', 'ВАНТАЖНИЙ АВТОМОБІЛЬ'\n",
    "        , 'MARITIMO', 'RAI', 'ЗАЛІЗНИЧНИЙ ВАГОН', 'MARITIMA', 'LCS-SEA', 'VESSEL, NON-CONTAINER'\n",
    "        , 'TRANSPORT PAR AIR', 'OTRO', 'ROAD', 'ACUÁTICA', 'MARITIME TRANSPORT', 'nan', 'AÉREA', 'OTHER'\n",
    "        , 'RAIL', 'COURIER (ADUANA AÉREA)', 'R', '30', 'VESSEL', 'FERROVIARIA'\n",
    "        , 'ВАНТАЖНИЙ АВТОМОБІЛЬ НА МОРСЬКОМУ СУДНІ', 'TENDIDO ELECTRICO (AEREO,SUBT)', 'BY SEA'\n",
    "        , 'MARÃ\\xadTIMO', 'MARÍTIMO', 'ПОШТОВЕ ВІДПРАВЛЕННЯ АВТОТРАНСПОРТОМ', 'ENCOMIENDA', 'INSTAL FIJAS'\n",
    "        , 'AEREA', 'LCS-ROAD', 'CONDUCTOR ELÉCTRICO', 'AEREO', 'SEA', 'FLUVIAL', 'MARITIME', 'OWN MEANS'\n",
    "        , '40', 'O', 'ТРУБОПРОВІДНИЙ ТРАНСПОРТ', '35', 'ICD', 'КОНТЕЙНЕР НА ВАНТАЖНОМУ АВТОМОБІЛІ'\n",
    "        , 'LAND - RAILWAY', 'TUBERIAS', '20', 'TERRESTRIAL', 'A', 'OTROS', 'S', 'AIR TRANSPORT', 'AÉREO'\n",
    "        , 'ROAD TRANSPORT', 'AERIEN', '25', 'CORREO', 'AÃ©REA', 'CONDUCTOR ELÃ©CTRICO', 'TRANSPORT PAR ROUTE'\n",
    "        , 'RAIL TRANSPORT', 'LCS', 'ICD-SEA', 'ACUÃ¡TICA', 'AÃ©REO', 'МОРСЬКЕ СУДНО', 'ICD-ROAD', 'TRUCK'\n",
    "        , 'TRANSPORT MARITIME', 'DUCTOS', 'КОНТЕЙНЕР НА ЗАЛІЗНИЧНОМУ ВАГОНІ', 'POSTAL', 'TERRESTRE'\n",
    "        , 'OTRAS VIAS', 'CARRETERA', 'CARRETERO', 'НЕВІДОМИЙ', 'ARREO', 'КОНТЕЙНЕР НА МОРСЬКОМУ СУДНІ'\n",
    "        , 'OTHERS', '10']\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa1d3e81-d211-4167-b455-af933ecaa164",
   "metadata": {},
   "source": [
    "# 5 上传的csv文件格式处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e94c173-3179-4e34-8d01-03c739342864",
   "metadata": {},
   "outputs": [],
   "source": [
    "# country_list = ['印度', '英国', '越南', '阿根廷', '埃塞俄比亚', '巴基斯坦', '俄罗斯陆运', '厄瓜多尔'\n",
    "#                 , '菲律宾', '哥伦比亚', '哥斯达黎加', '哈萨克斯坦', '加纳', '喀麦隆', '科特迪瓦', '肯尼亚'\n",
    "#                 , '莱索托', '马拉维', '美国', '孟加拉', '秘鲁', '秘鲁-海运', '秘鲁-空运', '墨西哥'\n",
    "#                 , '纳米比亚', '尼日利亚', '斯里兰卡', '坦桑尼亚', '乌干达', '乌克兰', '乌拉圭', '乌兹别克斯坦'\n",
    "#                 , '智利', '巴拉圭', '巴拿马', '博兹瓦纳']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68264b26-ba03-42f6-ba45-9d97eb44d2ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "country_list = ['俄罗斯陆运']\n",
    "\n",
    "# 获取文件目录，区分出口、进口数据\n",
    "from_path = Path('dataset/data_20250724/20251t3_csv_final_tmp/RUS')\n",
    "\n",
    "# file_tree = distinct_ie_data2(from_path, country_list)\n",
    "to_path = Path('dataset/data_20250724/20251t3_csv_final/final_files')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32fda405-c408-4bae-be37-f9f37ce5c745",
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "params_list = [([ctry], from_path, to_path, country36_dict) for ctry in country_list]\n",
    "\n",
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    # 提交多个任务\n",
    "    futures = [\n",
    "        executor.submit(lambda p: replace_tab_to_norm(*p), params)\n",
    "        for params in params_list\n",
    "    ]\n",
    "    # 按完成顺序获取结果\n",
    "    for future in as_completed(futures):\n",
    "        # logger.info(future.result().shape)\n",
    "        pass\n",
    "\n",
    "\n",
    "end = time.time()\n",
    "logger.info(f'replace_tab_to_norm takes {(end-start)/60} minutes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbd1e47f-22c5-493f-a48e-81e3a0a2cf76",
   "metadata": {},
   "outputs": [],
   "source": [
    "params_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "448c065b-b31d-4dec-92fd-3e1fde5eac9f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "87f16d9f-65de-4307-a342-38d8c5af3c98",
   "metadata": {},
   "source": [
    "# end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7eff7799-6f86-4cce-bf0b-239930474c91",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_end=time.time()\n",
    "logger.info(f'{(all_end-all_start)/60} minutes')\n",
    "logger.info('All done!\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f66f15f4-9e4d-4d1b-83db-6d268964ed9d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a6a0457-9e4d-4a0f-8591-09eaf9804a83",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
